/**
 * 词库爬虫之QQ输入法
 * Created by 小马哥 on 2017-04-16.
 */

// 设置中文字符集
phantom.outputEncoding="gbk";

var casper = require('casper').create({
    viewportSize: {
        width: 1024,
        height: 768
    }
});

/**
 * QQ输入法词库官网地址
 * @type {string}
 */
var qqThesaurusUrl = 'http://dict.qq.pinyin.cn/';
/**
 * 词库文件的后缀名
 * @type {string}
 */
var fileSuffix = '.qpyd';
/**
 * 词库文件下载路径
 * @type {string}
 */
var filePath = './thesaurus/qq/';

casper.start(qqThesaurusUrl);

casper.waitForSelector('#mContainer', function () {
    var categoryLinks = getCategoryLinks();
    var categoryLinksLen = categoryLinks.length;
    console.log('Category count is ' + categoryLinksLen);
    var pageCount = 1;
    var allPageArr = new Array();
    casper.start().each(categoryLinks, function (self, link) {
        // 先遍历各个分类，将分类下的页面收集起来
        self.thenOpen(link, function () {
            casper.echo('Current category is ' + casper.getTitle());
            var pageUrls = getPageUrls(link);
            var pageUrlsLen = pageUrls.length;
            casper.echo('Page url count is ' + pageUrlsLen);
            for(var i = 0; i < pageUrlsLen; i++){
                allPageArr.push(pageUrls[i]);
            }
            // 逐个打开分类下的页面，并下载词库文件
            if(pageCount == categoryLinksLen){
                openPages(allPageArr);
                return;
            }
            pageCount ++;
        });
    });
});
casper.run();

/**
 * 打开收集到的分类下的页面
 * @param pageUrls
 */
var openPages = function (pageUrls) {
    casper.start().each(pageUrls, function (self, link) {
        self.thenOpen(link, function () {
            casper.echo('Current page url is ' + link);
            // 从分页URL里面截取分类名称
            var categoryName = link.substring(link.indexOf('=') + 1, link.lastIndexOf('&'));
            casper.echo('Current category name is ' + categoryName);
            // 一个下载链接对应一个下载的文件名
            var downloadNames = getDownloadNames();
            console.log('Download name count：' + downloadNames.length);
            var downloadLinks = getDownloadLinks();
            var downloadLinksLen = downloadLinks.length;
            console.log('Download url count：' + downloadLinksLen);
            for(var i = 0; i < downloadLinksLen; i++){
                var filePrefix = downloadNames[i];
                if(filePrefix.indexOf('/') != -1){
                    // 用“_”代替名称中的“/”，避免出现文件夹
                    filePrefix = filePrefix.replace(/\//g, '_');
                }else if(filePrefix.indexOf('\\') != -1){
                    // 用“_”代替名称中的“\”，避免出现文件夹
                    filePrefix = filePrefix.replace(/\\/g, '_');
                }
                var fileName = filePath + categoryName + '/' + filePrefix + fileSuffix;
                var downloadUrl = downloadLinks[i];
                casper.echo('->FileName is ' + fileName);
                casper.echo('->Download url is ' + downloadUrl);
                casper.download(downloadUrl, fileName);
            }
        });
    });
};

/**
 * 当前分类下需要跳转的页面的URL集合
 * @param link
 * @returns {Array}
 */
var getPageUrls = function (link) {
    var pageNum = getPageNum();
    casper.echo('Page num is ' + pageNum);
    var result = new Array();
    for(var i = 1; i <= pageNum; i++){
        result.push(link + '&page=' + i);
    }
    return result;
};

/**
 * 获取当前分类下的页数
 * 从“尾页”指向的URL里面截取
 * @returns {Object}
 */
var getPageNum = function () {
    return casper.evaluate(function () {
        var children = $('#cikuCategoryPage').find('.last-page-active');
        if(children.length != 2){
            return 1;
        }
        var lastUrl = $(children[children.length - 1]).attr('href');
        return lastUrl.substring(lastUrl.lastIndexOf('=') + 1, lastUrl.length);
    });
};

/**
 * 获取当前页面下的词库的下载连接的集合
 * @returns {Object}
 */
var getDownloadLinks = function () {
    return casper.evaluate(function () {
        var result = new Array();
        var children = $('#cikuCategoryPage').find('.downloadICO');
        var childrenLen = children.length;
        for(var i = 0; i < childrenLen; i++){
            result.push($(children[i]).attr('href'));
        }
        return result;
    });
};

/**
 * 获取需要下载的词库对应的文件名
 * @returns {Object}
 */
var getDownloadNames = function () {
    return casper.evaluate(function () {
        var result = new Array();
        var children = $('#cikuCategoryPage').find('.cickSummary');
        var childrenLen = children.length;
        for(var i = 0; i < childrenLen; i++){
            result.push($(children[i]).find('.title').find('a').text());
        }
        return result;
    });
};

/**
 * 获取id为dict_category_show的div下的样式为dict_category_list_title的a标签
 * 取出a标签的href属性，放入数组返回
 */
var getCategoryLinks = function () {
    return casper.evaluate(function () {
        var result = new Array();
        var children = $('#mContainer').find('.cikuCategory');
        var childrenLen = children.length;
        // QQ输入法官网
        var qqOfficialUrl = 'http://dict.qq.pinyin.cn';
        for(var i = 0; i < childrenLen; i++){
            result.push(qqOfficialUrl + $(children[i]).find('a').attr('href'));
        }
        return result;
    });
};

/**
 * 调试用
 * 单个文件下载
 */
var testSignalDownload = function () {
    var url = 'http://download.pinyin.qq.com/dict/download_cell.php?id=31670&name=Johnny\%27s%E4%BA%8B%E5%8A%A1%E6%89%80';
    var name = "./test/Johnny\'s事务所";
    casper.download(url, name + fileSuffix);
};

/**
 * 调试用
 * 截取当前网页的快照
 */
var captureCurrentHtml = function () {
    casper.captureSelector('dict_category_show.png', 'html');
};

/**
 * 调试用
 * 调试根div下是否有内容
 */
var debugDictCategoryShowHtml = function () {
    var categoryHtml = casper.evaluate(function () {
        // return document.getElementById('dict_category_show').innerHTML;
        return $('#dict_category_show').html();
    });
    casper.echo(categoryHtml);
};
